package convertfiles.helper;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.Charset;

public class CrawlerHelper {
	public static String getContentFromUrl(String url) {
		String content = "";
		try {
			URL dataURL = new URL(url);
	        HttpURLConnection connection = (HttpURLConnection) dataURL.openConnection();
	        connection.setReadTimeout(500000);
	        connection.setConnectTimeout(1000000);
	        connection.setRequestProperty( "User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.21) Gecko/20110830 AlexaToolbar/alxf-2.14 AlexaToolbar/pXVUnvuf-1.2 Firefox/3.6.21");
	        connection.setRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
	        connection.setRequestProperty("Accept-Language", "en-us,en;q=0.5");
	        connection.setRequestProperty("Accept-Charset", "    ISO-8859-1,utf-8;q=0.7,*;q=0.7");
	        connection.setRequestProperty("Connection", "keep-alive");
	        connection.setRequestProperty("Keep-Alive", "115");
	        connection.setRequestProperty("Cache-Control", "max-age=0");
	        BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream(), Charset.forName("UTF8")));
	        while (true) {
	            String str = reader.readLine();
	            if (str == null)
	                break;
	            content += str;
	        }
		} catch (Exception e) {
			// TODO: handle exception
			e.printStackTrace();
		}
		return content;
	}
}
